# coding:utf-8
import os
import shutil
import sqlite3
import sys
import time

from bs4 import BeautifulSoup

ROOT = r"E:\down_data\gzlg_botu"

cur_dir_fullpath = os.path.dirname(os.path.abspath(__file__))
StartTime = time.time()

DATAROOT = r"E:\down_data\gzlg_botu"
DB3TEMPLATE = r'D:\Python3Project\120\data\botu_gzlg\ref\template.db3'

DB3FILE = os.path.join(r'D:\Python3Project\120\data\botu_gzlg\ouput',
                       'gitbotubook_' + time.strftime('%Y%m%d', time.localtime(time.time())) + '.db3')


def insertData(sqlList):
    conn = sqlite3.connect(DB3FILE)
    cur = conn.cursor()
    for sql in sqlList:
        try:
            cur.execute(sql)
        except:
            print(sql)
            sys.exit(-1)
    conn.commit()
    conn.close()


def get_data(parent, filename):
    file_root = os.path.join(parent, filename)
    print(file_root)
    lst = file_root.split('\\')
    # print(lst)
    rawid = filename[:-5]
    cover = ''
    coverpath = os.path.join(r'D:\Python3Project\120\data\botu_gzlg\coverout', lst[-2], filename[:-5].lower() + '.jpg')
    # print(coverpath)
    if os.path.exists(coverpath):
        cover = '/smartlib/gitbotubook/' + lst[-2] + '/' + filename[:-5].lower() + '.jpg'
    # print(cover)
    lngid = 'GIT_BOTU_TS_' + str(rawid)
    batch = time.strftime('%Y%m%d', time.localtime(time.time()))
    batch = batch + '00'
    language = 'EN'
    country = 'US'
    provider = 'gitbotubook'
    type = '1'
    medium = '2'
    date = "1900"
    date_created = "19000000"
    provider_url = 'gitbotubook@http://www.cnbooksearch.com/BookRead.aspx?bookid=' + str(rawid)
    provider_id = 'gitbotubook@' + str(rawid)
    f = open(file_root, 'r', encoding="utf-8")
    htmlText = f.read()
    soup = BeautifulSoup(htmlText, 'lxml')
    title = ""
    identifier_pisbn = ""
    identifier_eisbn = ""
    publisher = ""
    creator = ""
    provider_subject = ""
    try:
        div = soup.find("div", id="token")
        tas = div.find_all('a')
    except:
        print(file_root)
        return
    for a in tas[1:]:
        provider_subject = provider_subject + a.text + ";"
    provider_subject = provider_subject[:-1]
    provider_subject = provider_subject.replace("'", '"')
    b = soup.find('b', id="bookname")
    if not b:
        return
    title = b.text
    title = title.replace("'", '"').replace('\n', "").replace('\r', '').replace("\t", "").replace("\0", "")
    if title == "":
        # print(rawid)
        return
    # print(title)
    p = soup.find('p', id="author")
    try:
        if not p:
            creator = ''
        else:
            author = p.find('a')
            creator = author.text
            creator = creator.replace("'", '"')
            creator = creator.replace(",", ";")
    except:
        return
    isbn = soup.find('p', id='isbn')

    if not isbn:
        identifier_pisbn = ""
    else:
        isbn = isbn.text
        isbn = isbn.split(':')
        isbn_1 = isbn[1]
        isbn_1 = isbn_1.replace("'", '"').replace("-", "").replace("/", ";").replace(", ", ";").replace("—",
                                                                                                        "").replace(
            "--", "").replace("-", "").replace(" (", "(").replace(" ", ";")

    pisbn = isbn_1.split(";")
    # print(isbn_1)
    if isbn_1.find("(Print)") > 0:
        # print(isbn_1)
        for isbndata in pisbn:
            if isbndata.find("(Print)") > 0:
                identifier_pisbn = identifier_pisbn + isbndata.replace("(Print)", "") + ";"
            if isbndata.find("(Online)") > 0:
                identifier_eisbn = identifier_eisbn + isbndata.replace("(Online)", "") + ";"
    else:
        # print(isbn_1)
        identifier_pisbn = isbn_1
    identifier_pisbn = identifier_pisbn.strip(";")
    # print(identifier_eisbn+"___"+identifier_pisbn)
    press = soup.find('p', id='press')
    if not press:
        publisher = ""
    else:
        press = press.text
        press = press.split(':')
        publisher = press[1]
        publisher = publisher.replace("'", '"')

    sql = "insert into modify_title_info_zt(identifier_eisbn,publisher,lngid,rawid,language,country ,identifier_pisbn,provider,type ,medium ,batch, title,provider_url, provider_id ,creator,cover,provider_subject,date,date_created) values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % (
    identifier_eisbn, publisher, lngid, rawid, language, country, identifier_pisbn, provider, type, medium, batch,
    title, provider_url, provider_id, creator, cover, provider_subject, date, date_created)
    return sql


if __name__ == "__main__":
    shutil.copyfile(DB3TEMPLATE, DB3FILE)  # oldfile和newfile都只能是文件

    html_root = os.path.join(DATAROOT, 'html')
    sqlList = []
    cnt = 0
    for parent, dirnames, filenames in os.walk(html_root):
        for filename in filenames:
            if not filename.endswith('.html'):
                continue
            cnt += 1
            sql = get_data(parent, filename)
            if sql == None:
                continue
            sqlList.append(sql)
            if cnt % 100 == 0:
                print(cnt)
                insertData(sqlList)
                sqlList = []
        # print('time total:%.2fs' % (time.time()-StartTime))
        insertData(sqlList)
        sqlList = []
    print('time total:%.2fs' % (time.time() - StartTime))
